package com.putao.spider;

import java.util.ArrayList;
import java.util.List;

import org.apache.commons.collections.CollectionUtils;

import com.putao.Server;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

public class ZhihuCatSpider implements PageProcessor {

	private Site site = Site.me().setRetryTimes(3).setSleepTime(5000);
	Integer count = 0;
	List<String> topicList = new ArrayList<>();
	
	Server server;

	public ZhihuCatSpider() {
		super();
		server = new Server();
	}
	@Override
	public void process(Page page) {
		// 筛选出分类id数据
		List<String> ids = page.getHtml()
				.xpath("//ul[@class=\"zm-topic-cat-main clearfix\"]/li[@class=\"zm-topic-cat-item\"]/@data-id").all();
		List<String> names = page.getHtml()
				.xpath("//ul[@class=\"zm-topic-cat-main clearfix\"]/li[@class=\"zm-topic-cat-item\"]/a/text()").all();
		if (CollectionUtils.isNotEmpty(ids)) {
			for (int i = 0; i < ids.size(); i++) {
				server.batchAddTopic(ZhihuTopicPost.post(ids.get(i),names.get(i)));
			}
		}
	}
/*	@Override
	public void process(Page page) {
		// 筛选出分类id数据
		List<String> ids = page.getHtml()
				.xpath("//ul[@class=\"zm-topic-cat-main clearfix\"]/li[@class=\"zm-topic-cat-item\"]/@data-id").all();
		if (CollectionUtils.isNotEmpty(ids)) {
			for (String id : ids) {
				topicList.addAll(ZhihuTopicPost.post(id));
			}
		}
		Spider.create(new ZhihuSpider()).addUrl((String[]) topicList.toArray())
		.addPipeline(new JsonFilePipeline("D:\\webmagic\\")).addPipeline(new ConsolePipeline()).thread(5).run();
	}
*/
	@Override
	public Site getSite() {
		site.setCharset("UTF-8");
		return site;
	}

	public static void main(String[] args) {
		Spider.create(new ZhihuCatSpider()).addUrl("https://www.zhihu.com/topics")
				.addPipeline(new JsonFilePipeline("D:\\webmagic\\")).addPipeline(new ConsolePipeline()).thread(1).run();
	}
}